#include <string>
#include <fstream>
#include <list>
#include <iconv/iconv.h>
#include <curl/curl.h>
#include <curl/easy.h>
#include <libxml/tree.h>
#include <libxml/uri.h>
#include <libxml/HTMLparser.h>
#include <libxml/HTMLTree.h>

using namespace std;

#define MAX_FILE_SIZE 1024*1024

typedef struct _MONITOR_INFO
{
    string      szDomain;
    string      szUrl;
    string      szTitle;
}
MONITOR_INFO, *PMONITOR_INFO;

typedef struct _URL_INFO
{
    string      szUrl;
    bool        bProcessed;
}
URL_INFO, *PURL_INFO;

size_t DataFunction(char *data, size_t size, size_t nmemb, std::string *buffer)
{
    if (buffer == NULL)
        return 0;
    buffer->append(data, size*nmemb);
    if (buffer->size() < MAX_FILE_SIZE)
        return size*nmemb;
    else
        return 0;
}

void GetAllNodesByName(xmlNodePtr node, const string& name, list<xmlNodePtr>& nodes)
{
    xmlNodePtr cur_node = NULL;
    for (cur_node = node; cur_node; cur_node = cur_node->next)
    {
        if (cur_node->name && xmlStrcmp(cur_node->name, (const xmlChar*)name.c_str()) == 0)
            nodes.push_back(cur_node);
        GetAllNodesByName(cur_node->children, name, nodes);
    }
}

void GetNodeText(xmlNodePtr node, string& text)
{
    xmlNodePtr cur_node = NULL;
    for (cur_node = node; cur_node; cur_node = cur_node->next)
    {
        if (xmlNodeIsText(cur_node) && !xmlIsBlankNode(cur_node) && cur_node->content)
            text += (const char*)cur_node->content;
        GetNodeText(cur_node->children, text);
    }
}

void GetContentCharsetFromResponse(const string& szResponse, string& szCharset)
{
    string szResponseCopy(szResponse);
    _strlwr((char*)szResponseCopy.c_str());
    string::size_type pos = szResponseCopy.rfind("\r\ncontent-type: ");
    if (pos == string::npos)
        return;
    const char* p1 = szResponseCopy.c_str() + pos + strlen("\r\ncontent-type: ");
    const char* p2 = strstr(p1, " charset=");
    if (!p2)
        return;
    p2 += strlen(" charset=");
    while(*p2 && *p2 != ';' && *p2 != ' ' && *p2 != '\r' && *p2 != '\n' && *p2 != '\t')
    {
        szCharset += *p2;
        p2++;
    }
}

void GetContentTypeFromResponse(const std::string& szResponse, std::string& szContentType)
{
    string szResponseCopy(szResponse);
    _strlwr((char*)szResponseCopy.c_str());
    string::size_type pos = szResponseCopy.rfind("\r\ncontent-type: ");
    if (pos == string::npos)
        return;
    const char* p = szResponseCopy.c_str() + pos + strlen("\r\ncontent-type: ");
    while(*p && *p != ';' && *p != ' ' && *p != '\r' && *p != '\n' && *p != '\t')
    {
        szContentType += *p;
        p++;
    }
}

void GetLocationFromResponse(const string& szResponse, string& szLocation)
{
    string szResponseCopy(szResponse);
    _strlwr((char*)szResponseCopy.c_str());
    string::size_type pos = szResponseCopy.rfind("\r\nlocation: ");
    if (pos == string::npos)
        return;
    const char* p1 = szResponseCopy.c_str() + pos + strlen("\r\nlocation: ");
    while(*p1 && *p1 != ';' && *p1 != ' ' && *p1 != '\r' && *p1 != '\n' && *p1 != '\t')
    {
        szLocation += *p1;
        p1++;
    }
}

void GetContentCharsetFromResource(const string& szResource, string& szCharset)
{
    htmlDocPtr htmlDoc = htmlReadMemory(szResource.c_str(), szResource.size(), NULL, NULL, HTML_PARSE_RECOVER | HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING);
    if (htmlDoc)
    {
        const xmlChar* enc = htmlGetMetaEncoding(htmlDoc);
        if (enc)
        {
            while (*enc && *enc != ';' && *enc != ' ' && *enc != '\r' && *enc != '\n' && *enc != '\t')
            {
                szCharset += *enc;
                enc++;
            }
        }
        xmlFreeDoc(htmlDoc);
    }
}

string PopUrl(list<URL_INFO>& urls_info)
{
    for (list<URL_INFO>::iterator iter = urls_info.begin(); iter != urls_info.end();iter++ )
    {
        if (!iter->bProcessed)
        {
            iter->bProcessed = true;
            return iter->szUrl;
        }
    }
    return "";
}

void PushUrl(list<URL_INFO>& urls_info, string szUrl)
{
    bool bFinded = false;
    for (list<URL_INFO>::iterator iter = urls_info.begin();
         iter != urls_info.end() && !bFinded;
         iter++ )
    {
        if (_stricmp(iter->szUrl.c_str(), szUrl.c_str()) == 0)
            bFinded = true;
    }
    if (!bFinded)
    {
        URL_INFO url_info;
        url_info.szUrl = szUrl;
        url_info.bProcessed = false;
        urls_info.push_back(url_info);
    }
}

size_t GetCount(list<URL_INFO>& urls_info, bool bState)
{
    size_t count = 0;
    for (list<URL_INFO>::iterator iter = urls_info.begin();
        iter != urls_info.end();
        iter++ )
    {
        if (iter->bProcessed == bState)
            count++;
    }
    return count;
}

void PushUrl(list<MONITOR_INFO>& monitors_info, MONITOR_INFO& monitor_info)
{
    bool bFinded = false;
    xmlURIPtr uri = xmlParseURI(monitor_info.szUrl.c_str());
    if (uri && uri->server)
    {        
        for (list<MONITOR_INFO>::iterator iter = monitors_info.begin();
            iter != monitors_info.end() && !bFinded;
            iter++ )
        {
            if (_stricmp(iter->szDomain.c_str(), uri->server) == 0)
                bFinded = true;
        }
        if (!bFinded)
        {
            monitor_info.szDomain = uri->server;
            monitors_info.push_back(monitor_info);
        }
        xmlFreeURI(uri);
    }
}

int main(int argc, char* argv[])
{
    list<MONITOR_INFO> monitors_info;
    list<URL_INFO> urls_info;
    string szCurrentUrl;
    string szPrevUrl;
    unsigned long ulMaxUrlCount = 0;
    unsigned long ulMaxDownCount = 0;
    string szRootServer;
    double dTotalSizeDownload = 0;
    double dTotalSpeedDownload = 0;
    double dTotalTimeDownload = 0;

    printf("\nConsole URL Fetcher v1.0\n");
    printf("(C)2007 by Sergey Markin/W3Monitor Co.\n\n");
    if (argc < 4)
    {
        printf("Usage: urlfetcher.exe {URL} {Max external URL} {Max downloads}\n\n");
        return 0;
    }

    xmlInitParser();

    xmlURIPtr serverUri = xmlParseURI(argv[1]);
    if (!serverUri || !serverUri->server)
    {
        printf("Invalid input URL format\n");
        if (serverUri)
            xmlFreeURI(serverUri);
        xmlCleanupParser();
        return 0;
    }

    ulMaxUrlCount = atol(argv[2]);
    ulMaxDownCount = atol(argv[3]);    

    szCurrentUrl = argv[1];

    szRootServer = serverUri->server;
    string szServer(serverUri->server);
    string::size_type pos = szServer.rfind(".");
    if (pos != string::npos)
    {
        pos = szServer.rfind(".", pos-1);
        if (pos != string::npos)
        {
            szRootServer = szServer.substr(pos+1);
        }
    }

    curl_global_init(CURL_GLOBAL_ALL);

    while ( szCurrentUrl.size() &&
            (monitors_info.size() < ulMaxUrlCount || ulMaxUrlCount == 0) &&
            (GetCount(urls_info, true) < ulMaxDownCount || ulMaxDownCount == 0))
    {
        CURLcode curl_code = CURLE_OK;
        CURL *curl_handle = NULL;        
        unsigned long ulResponseCode = 0;
        string szCurlMessage;
        string szData, szHeader;
        double dSizeDownload = 0;
        double dSpeedDownload = 0;
        double dTimeDownload = 0;

        curl_handle = curl_easy_init();

        curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 1);
        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0);
        curl_easy_setopt(curl_handle, CURLOPT_MAXFILESIZE, MAX_FILE_SIZE);
        curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, DataFunction);
        curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &szData);
        curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, DataFunction);
        curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, &szHeader);
        curl_easy_setopt(curl_handle, CURLOPT_URL, szCurrentUrl.c_str());
        curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 6.0; Win32)");        
        if (szPrevUrl.size())
            curl_easy_setopt(curl_handle, CURLOPT_REFERER, szPrevUrl.c_str());
        else
            curl_easy_setopt(curl_handle, CURLOPT_AUTOREFERER, 1);
        //curl_easy_setopt(curl_handle, CURLOPT_PROXY, "10.32.0.2:8080");
        //curl_easy_setopt(curl_handle, CURLOPT_PROXYUSERPWD, "sergius:*****");
        curl_easy_setopt(curl_handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY);

        printf("Start download from: %s\n", szCurrentUrl.c_str());
        curl_code = curl_easy_perform(curl_handle);

        curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &ulResponseCode);
        szCurlMessage = curl_easy_strerror(curl_code);
        curl_easy_getinfo(curl_handle, CURLINFO_SIZE_DOWNLOAD, &dSizeDownload);
        curl_easy_getinfo(curl_handle, CURLINFO_SPEED_DOWNLOAD, &dSpeedDownload);
        curl_easy_getinfo(curl_handle, CURLINFO_TOTAL_TIME, &dTimeDownload);

        dTotalSizeDownload += dSizeDownload;
        dTotalSpeedDownload += dSpeedDownload;
        dTotalTimeDownload += dTimeDownload;

        if ((curl_code == CURLE_OK && ulResponseCode == 200) ||
            (curl_code == CURLE_PARTIAL_FILE && dSizeDownload > 0 && ulResponseCode == 200))
        {
            char *szEffectiveUrl = NULL;
            string szCharset;
            string szContentType;
            htmlDocPtr htmlDoc = NULL;
            
            printf("Download completed\n");
            
            curl_easy_getinfo(curl_handle, CURLINFO_EFFECTIVE_URL, &szEffectiveUrl);

            GetContentCharsetFromResponse(szHeader, szCharset);
            if (!szCharset.size())
            {
                GetContentCharsetFromResource(szData, szCharset);
            }

            GetContentTypeFromResponse(szHeader, szContentType);
            if (szContentType == "text/html")
            {
                if (!szCharset.size())
                    htmlDoc = htmlReadMemory(szData.c_str(), szData.size(), szCurrentUrl.c_str(), NULL, HTML_PARSE_RECOVER | HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_NONET);
                else
                    htmlDoc = htmlReadMemory(szData.c_str(), szData.size(), szCurrentUrl.c_str(), szCharset.c_str(), HTML_PARSE_RECOVER | HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_NONET);            

                if (htmlDoc)
                {
                    list<xmlNodePtr> alist;
                    GetAllNodesByName(xmlDocGetRootElement(htmlDoc), "a", alist);
                    for (list<xmlNodePtr>::iterator alist_iter = alist.begin();
                        alist_iter != alist.end() && (monitors_info.size() < ulMaxUrlCount || ulMaxUrlCount == 0);
                        alist_iter++)
                    {
                        xmlChar* href = xmlGetProp(*alist_iter, (const xmlChar*)"href");
                        if (href)
                        {
                            xmlChar* normalUrl = NULL;
                            normalUrl = xmlBuildURI(href, (const xmlChar*)szEffectiveUrl);
                            if (normalUrl)
                            {
                                xmlURIPtr uri = xmlParseURI((const char*)normalUrl);
                                if (uri && uri->server && uri->scheme && (strcmp(uri->scheme, "http") == 0 || strcmp(uri->scheme, "https") == 0))
                                {
                                    if (strstr(uri->server, szRootServer.c_str()) != NULL)
                                    {
                                        PushUrl(urls_info, (const char*)normalUrl);
                                    }
                                    else
                                    {
                                        MONITOR_INFO monitor_info;
                                        monitor_info.szUrl = (const char*)normalUrl;
                                        GetNodeText((*alist_iter)->children, monitor_info.szTitle);
                                        PushUrl(monitors_info, monitor_info);
                                    }
                                    xmlFreeURI(uri);
                                }
                                xmlFree(normalUrl);
                            }                        
                            xmlFree(href);
                        }
                    }
                    xmlFreeDoc(htmlDoc);
                }
                else
                {
                    printf("Parsing HTML error\n");
                }                
            }
            else
            {
                printf("Unsupported content type: %s\n", szContentType.c_str());
            }
        }
        else
        if ((curl_code == CURLE_OK && (ulResponseCode == 301 || ulResponseCode == 302)) ||
            (curl_code == CURLE_PARTIAL_FILE && dSizeDownload > 0 && (ulResponseCode == 301 || ulResponseCode == 302)))
        {
            string szLocation;
            GetLocationFromResponse(szHeader, szLocation);
            if (szLocation.size())
            {
                xmlURIPtr uri = xmlParseURI((const char*)szLocation.c_str());
                if (uri && uri->server && uri->scheme && (strcmp(uri->scheme, "http") == 0 || strcmp(uri->scheme, "https") == 0))
                {
                    if (strstr(uri->server, szRootServer.c_str()) != NULL)
                    {
                        PushUrl(urls_info, szLocation.c_str());
                        printf("Redirect to: %s\n", szLocation.c_str());
                    }
                    else
                    {
                        printf("Redirect to another server forbidden. URL: %s\n", szLocation.c_str());
                    }
                    xmlFreeURI(uri);
                }
            }
        }
        else
        {
            printf("Download error. Curl code: %u. Curl message: %s, Response code: %u\n", curl_code, szCurlMessage.c_str(), ulResponseCode);
        }

        curl_easy_cleanup(curl_handle);
        
        szPrevUrl = szCurrentUrl;
        szCurrentUrl = PopUrl(urls_info);
    }

    xmlDocPtr xmlDoc = xmlNewDoc((const xmlChar*)"1.0");
    if (xmlDoc)
    {
        xmlDocSetRootElement(xmlDoc, xmlNewNode(NULL, (const xmlChar*)"monitors"));
        for (list<MONITOR_INFO>::iterator info_iter = monitors_info.begin(); info_iter != monitors_info.end(); info_iter++)
        {
            xmlNodePtr monitor = xmlNewNode(NULL, (const xmlChar*)"monitor");
            xmlSetProp(monitor, (const xmlChar*)"url", (const xmlChar*)info_iter->szUrl.c_str());
            xmlSetProp(monitor, (const xmlChar*)"title", (const xmlChar*)info_iter->szTitle.c_str());
            xmlAddChild(xmlDocGetRootElement(xmlDoc), monitor);
        }
        printf("\nSave output XML file ");
        int res = xmlSaveFormatFileEnc("urlfetcher.xml", xmlDoc, "utf-8", 1);
        if (res == -1)
        {
            printf("failed\n");
        }
        else
        {
            printf("completed\n");
        }
        xmlFreeDoc(xmlDoc);
    }    

    xmlFreeURI(serverUri);

    printf("\nSummary URL Info\nExternal URLs: %u\nInternal URLs: %u\nDownloads: %u\n\n",
            monitors_info.size(),
            urls_info.size(),
            GetCount(urls_info, true));
    printf("Summary Download Info\nTotal size: %.0f bytes\nAverage Speed: %.0f kbytes per second\nTotalTime: %.0f secons\n",
            dTotalSizeDownload,
            dTotalSpeedDownload / GetCount(urls_info, true) / 1024,
            dTotalTimeDownload);

    curl_global_cleanup();
    xmlCleanupParser();
    return 0;
}
